import tensorflow as tf
from tensorflow.keras import datasets
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
import plotly
# https://stackoverflow.com/questions/57658935/save-jupyter-notebook-with-plotly-express-widgets-displaying
plotly.offline.init_notebook_mode()
(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
Downloading data from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz 170500096/170498071 [==============================] - 365s 2us/step
# https://stackoverflow.com/questions/51235508/in-which-folder-on-pc-windows-10-does-load-data-save-a-dataset-in-keras
! ls ~/.keras/datasets/cifar-10-batches-py
batches.meta data_batch_2 data_batch_4 readme.html data_batch_1 data_batch_3 data_batch_5 test_batch
train_images.shape
(50000, 32, 32, 3)
... provides convenient indexing expressivityimage_index = 0
train_images[image_index].shape, train_images[image_index,...].shape
((32, 32, 3), (32, 32, 3))
np.newaxis provides convenient dimensionality expressivity# https://stackoverflow.com/questions/17394882/how-can-i-add-new-dimensions-to-a-numpy-array
train_images[np.newaxis,image_index,...].shape
(1, 32, 32, 3)
# https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/images/cnn.ipynb#scrollTo=K3PAELE2eSU9
def plot_cifar10(x,y):
class_names = ['airplane', 'automobile', 'bird', 'cat', 'deer',
'dog', 'frog', 'horse', 'ship', 'truck']
plt.subplot(111);plt.xticks([]);plt.yticks([]);plt.grid(False)
plt.imshow(x, cmap=plt.cm.binary);plt.xlabel(y)
plot_cifar10(train_images[image_index],class_names[train_labels[image_index][0]])
rgb = {v:k for k,v in enumerate("rgb")}
print(rgb)
image_index = 0
train_images[image_index][..., rgb['r']]
{'r': 0, 'g': 1, 'b': 2}
array([[ 59, 43, 50, ..., 158, 152, 148],
[ 16, 0, 18, ..., 123, 119, 122],
[ 25, 16, 49, ..., 118, 120, 109],
...,
[208, 201, 198, ..., 160, 56, 53],
[180, 173, 186, ..., 184, 97, 83],
[177, 168, 179, ..., 216, 151, 123]], dtype=uint8)
# https://plotly.com/python/3d-scatter-plots/
# https://plotly.com/python/colorscales/
# https://plotly.com/python/creating-and-updating-figures/
# https://community.plotly.com/t/plotly-express-multiple-plots-overlay/31984
def plot_channel(x, h, color, alpha, loc, i=[0], j=[0]):
i_grid,j_grid = np.meshgrid(*[range(i) for i in x[0].shape[:2]])
df = pd.DataFrame(columns=['i','j','h','c'])
for x_,h_,i_,j_ in zip(x,h,i,j):
tmp = pd.DataFrame({'i': i_+i_grid.ravel(), 'j': j_+j_grid.ravel(), 'h': h_+0*i_grid.ravel()})
tmp['c'] = x_.ravel()
df = df.append(tmp)
fig.append_trace(go.Scatter3d(x=df.i, y=df.j, z=df.h, opacity=alpha,
mode='markers', marker={'color':df.c, 'colorscale': color}),*loc)
# a few other helpful posts which ultimately led to my solution above
# https://community.plotly.com/t/specifying-a-color-for-each-point-in-a-3d-scatter-plot/12652
# https://www.reddit.com/r/rstats/comments/g3tulu/is_it_possible_to_vary_alphaopacity_by_group_in_a/
# https://plotly.com/python/3d-scatter-plots/
# .. mode='markers', marker=dict(color='(255,0,0)', size=10) ...
# https://stackoverflow.com/questions/53875880/convert-a-pandas-dataframe-of-rgb-colors-to-hex
# https://stackoverflow.com/questions/46750462/subplot-with-plotly-with-multiple-traces
fig = plotly.subplots.make_subplots(rows=1,cols=2, specs=[[{'type': 'scene'}, {'type': 'scene'}]])
plot_channel([train_images[image_index][..., rgb['r']]/255], [0], 'Reds', 0.33, [1,1])
plot_channel([train_images[image_index][..., rgb['g']]/255], [1], 'Greens', 0.33, [1,1])
plot_channel([train_images[image_index][..., rgb['b']]/255], [2], 'Blues', 0.33, [1,1])
plot_channel([train_images[image_index][..., rgb['r']]/255,
train_images[image_index][..., rgb['g']]/255,
train_images[image_index][..., rgb['b']]/255], [0,1,2], 'Greys', 0.33, [1,2], [0]*3, [0]*3)
fig.show()
number_kernels = 10
kernel_width = 5
kernels = np.random.rand(number_kernels*kernel_width*kernel_width)
kernels = kernels.reshape((number_kernels,kernel_width,kernel_width))-0.5
print(kernels.shape)
kernels
(10, 5, 5)
array([[[ 0.39773621, -0.22900528, 0.09881262, 0.39870386,
0.2217557 ],
[ 0.09521832, -0.0603677 , 0.07556154, -0.03685071,
-0.2009564 ],
[-0.23324775, 0.13597168, -0.38572881, 0.17755535,
0.02958676],
[ 0.17361706, -0.28860532, 0.19120053, -0.25548537,
-0.03434457],
[ 0.43822891, -0.17325585, 0.27953384, -0.40592455,
0.22843153]],
[[-0.25219432, 0.32409323, 0.05650003, -0.35506418,
-0.21340797],
[-0.23070086, 0.23789205, -0.113693 , -0.12188067,
-0.28537089],
[ 0.35086779, -0.44085513, 0.18621295, 0.15655813,
-0.08871664],
[ 0.02848524, -0.0364151 , -0.2990787 , -0.46341008,
0.09617642],
[ 0.26849829, -0.36769018, -0.22750486, -0.42586508,
0.35661221]],
[[ 0.41166825, 0.28066959, 0.34855888, 0.15640564,
0.10742895],
[-0.04649512, -0.096581 , 0.05806068, -0.13888157,
0.32628453],
[ 0.12112283, -0.41475131, 0.4523904 , 0.13580666,
-0.22023767],
[-0.16963556, 0.44228029, 0.42031119, 0.41571177,
-0.4084773 ],
[-0.31393954, -0.00194135, 0.19085985, 0.33623604,
-0.33100384]],
[[ 0.4522182 , 0.10008379, 0.13888906, 0.03366733,
0.05187882],
[ 0.49325739, 0.49173976, -0.47655801, -0.04603961,
0.01312172],
[ 0.06674941, 0.12325937, 0.4237374 , 0.22924355,
0.09154674],
[-0.46924761, -0.18808748, -0.26755613, -0.081073 ,
-0.20719813],
[-0.26687857, 0.11433975, -0.11202769, -0.49593877,
-0.34071989]],
[[ 0.35583374, 0.05417797, 0.19429293, 0.04690031,
-0.18076017],
[-0.26390998, 0.40771601, -0.18751396, -0.42328216,
0.4092391 ],
[ 0.20935034, -0.46515889, -0.38332817, -0.45173748,
0.25107204],
[-0.00946868, -0.21123139, -0.09209932, 0.24485565,
-0.36954314],
[ 0.42544599, -0.35006576, 0.08926736, 0.02709029,
-0.36106572]],
[[ 0.1917415 , 0.12923727, -0.09677804, 0.00235128,
0.22352569],
[-0.17820994, -0.11924904, 0.44308445, -0.21832577,
0.05272216],
[-0.04656538, -0.20568385, -0.03680397, -0.36465248,
0.22460203],
[ 0.44451258, 0.29328181, -0.38657691, -0.12423647,
-0.47787287],
[-0.27331273, -0.11125227, 0.42764733, -0.25131675,
-0.23771714]],
[[ 0.03610223, 0.39661193, 0.29929894, 0.09385738,
-0.47940092],
[ 0.10217362, 0.24694502, 0.27857437, 0.15733641,
-0.09451715],
[-0.11832424, 0.45136702, -0.47465776, 0.49306154,
-0.32778601],
[ 0.24639338, -0.46653094, -0.2127155 , -0.08344141,
0.42424004],
[ 0.44881008, 0.23767325, -0.08938394, 0.3618896 ,
-0.05306801]],
[[-0.33074493, -0.34806425, 0.03138761, 0.27801598,
0.39188293],
[-0.11004306, 0.00321699, -0.05659795, -0.11514728,
-0.44331126],
[-0.30439071, 0.256911 , -0.14817768, 0.27302665,
-0.05730085],
[-0.18186033, 0.04368706, -0.18500711, 0.12878156,
-0.17546038],
[-0.36264879, 0.4798238 , -0.10582065, -0.22548848,
0.45551775]],
[[ 0.11273648, -0.1755804 , -0.08959842, 0.47728864,
0.13929403],
[-0.45505838, 0.01442301, 0.49023557, -0.05229527,
-0.42875331],
[ 0.15720984, 0.36420407, -0.00366684, 0.37044311,
0.42731875],
[-0.06769426, -0.27619938, -0.13072469, 0.44154463,
0.34017505],
[ 0.48106994, 0.43965211, 0.095974 , -0.0406033 ,
-0.12272673]],
[[ 0.32695968, -0.0131281 , 0.33555404, 0.08094979,
-0.15762903],
[-0.07571525, 0.30388555, -0.04378966, -0.19016661,
-0.24949091],
[ 0.26619536, -0.19415484, -0.23356821, 0.31933752,
-0.1145632 ],
[ 0.37917397, 0.27573519, 0.4164153 , -0.16388523,
-0.21491698],
[-0.1297217 , -0.30005154, 0.09354702, -0.30788471,
-0.27059385]]])
image_index = 0
channel = 0
i,j = 0,0
train_images[image_index,
i:(i+kernel_width),
j:(j+kernel_width),
channel]
array([[ 59, 43, 50, 68, 98],
[ 16, 0, 18, 51, 88],
[ 25, 16, 49, 83, 110],
[ 33, 38, 87, 106, 115],
[ 50, 59, 102, 127, 124]], dtype=uint8)
kernel_index = 0
kernels[kernel_index,:,:]
array([[ 0.39773621, -0.22900528, 0.09881262, 0.39870386, 0.2217557 ],
[ 0.09521832, -0.0603677 , 0.07556154, -0.03685071, -0.2009564 ],
[-0.23324775, 0.13597168, -0.38572881, 0.17755535, 0.02958676],
[ 0.17361706, -0.28860532, 0.19120053, -0.25548537, -0.03434457],
[ 0.43822891, -0.17325585, 0.27953384, -0.40592455, 0.22843153]])
kernel_index = 0
image_index = 0
channel = 0
i,j = 0,0#1,1
(\
train_images[image_index,
i:(i+kernel_width),
j:(j+kernel_width),
channel] * kernels[kernel_index,:,:]\
).sum()
21.217480140740886
kernels[-1,:2,:]=-1
kernels[-1,2,:]=0
kernels[-1,3:,:]=1
kernels[-1,...]
array([[-1., -1., -1., -1., -1.],
[-1., -1., -1., -1., -1.],
[ 0., 0., 0., 0., 0.],
[ 1., 1., 1., 1., 1.],
[ 1., 1., 1., 1., 1.]])
kernels[-2,:,:2]=-1
kernels[-2,:,2]=0
kernels[-2,:,3:]=1
kernels[-2,...]
array([[-1., -1., 0., 1., 1.],
[-1., -1., 0., 1., 1.],
[-1., -1., 0., 1., 1.],
[-1., -1., 0., 1., 1.],
[-1., -1., 0., 1., 1.]])
convolutional_layer_output = np.zeros(list(np.array(train_images[image_index].shape[:2])-kernel_width+1)+[number_kernels])
print(convolutional_layer_output.shape)
for kernel_index in range(number_kernels):
for i in range(convolutional_layer_output.shape[0]):
for j in range(convolutional_layer_output.shape[1]):
convolutional_layer_output[i,j,kernel_index] = \
(kernels[kernel_index,:,:] *
(train_images[image_index,
i:(i+kernel_width),
j:(j+kernel_width),
channel]/255-0.5)).sum() # notice that this sums over color channels!
(28, 28, 10)
# https://matplotlib.org/3.3.3/api/_as_gen/matplotlib.pyplot.subplots.html
f, ((ax1, ax2, ax3), (ax4, ax5, ax6)) = plt.subplots(2, 3, figsize=(15,10))
ax1.imshow(convolutional_layer_output[...,-1])
ax2.imshow(train_images[image_index], cmap=plt.cm.binary)
ax3.imshow(convolutional_layer_output[...,-2])
ax4.imshow(kernels[-1,...])
ax5.axis('off')
ax6.imshow(kernels[-2,...])
<matplotlib.image.AxesImage at 0x7fdfa9bd5b80>
# might try to animate this later:
# https://plotly.com/python/animations/
# https://plotly.com/python/v3/gapminder-example/
fig = plotly.subplots.make_subplots(rows=1,cols=2, specs=[[{'type': 'scene'}, {'type': 'scene'}]])
plot_channel([train_images[image_index][..., rgb['r']]/255,
train_images[image_index][..., rgb['g']]/255,
train_images[image_index][..., rgb['b']]/255], [0,1,2], 'Greys', 0.33, [1,1], [0]*3, [0]*3)
for kern_indx in range(number_kernels):
plot_channel([convolutional_layer_output[...,kern_indx]/255], [3*kern_indx],
px.colors.named_colorscales()[kern_indx], 0.1, [1,2])
kernel_index=2
i,j=17,17
plot_channel([kernels[kernel_index,...]/255]*3, list(range(3)),
px.colors.named_colorscales()[kernel_index], 1, [1,1], i=[i]*3, j=[j]*3)
plot_channel([np.array([[0],[1],[0]]).dot(np.array([[0,1,0]]))], [3*kernel_index],
px.colors.named_colorscales()[kernel_index], 1, [1,2], i=[i], j=[j])
fig.show()
# https://stats.stackexchange.com/questions/335332/why-use-matrix-transpose-in-gradient-descent
pooling_width = 4
pooling_layer_result = np.zeros(list((np.array(convolutional_layer_output.shape[:2])/pooling_width).astype(int))+[convolutional_layer_output.shape[2]])
print(pooling_layer_result.shape)
i_pooling_layer_result = 0*pooling_layer_result
j_pooling_layer_result = 0*pooling_layer_result
for k in range(number_kernels):
for ii,i in enumerate(range(0, convolutional_layer_output.shape[1], pooling_width)):
for jj,j in enumerate(range(0, convolutional_layer_output.shape[1], pooling_width)):
pooling_layer_result[ii,jj,k] = convolutional_layer_output[i:(i+pooling_width),j:(j+pooling_width),k].max()
i_pooling_layer_result[ii,jj,k] = ii
j_pooling_layer_result[ii,jj,k] = jj
(7, 7, 10)
kern_indx = 8
tmp = [convolutional_layer_output[i:(i+pooling_width),j:(j+pooling_width),kern_indx]/255
for j in range(0,28,8) for i in range(4,28,8)]
tmp_i = [i for j in range(0,28,8) for i in range(4,28,8)]
tmp_j = [j for j in range(0,28,8) for i in range(4,28,8)]
tmp += [convolutional_layer_output[i:(i+pooling_width),j:(j+pooling_width),kern_indx]/255
for i in range(0,28,8) for j in range(4,28,8)]
tmp_i += [i for i in range(0,28,8) for j in range(4,28,8)]
tmp_j += [j for i in range(0,28,8) for j in range(4,28,8)]
jmp = [pooling_layer_result[i:(i+1),j:(j+1),kern_indx]/255
for j in range(0,7,2) for i in range(1,7,2)]
jmp_i = [i for j in range(0,7,2) for i in range(1,7,2)]
jmp_j = [j for j in range(0,7,2) for i in range(1,7,2)]
jmp += [pooling_layer_result[i:(i+1),j:(j+1),kern_indx]/255
for i in range(0,7,2) for j in range(1,7,2)]
jmp_i += [i for i in range(0,7,2) for j in range(1,7,2)]
jmp_j += [j for i in range(0,7,2) for j in range(1,7,2)]
kern_indx = 2
tmp = [tmp]
tmp_i = [tmp_i]
tmp_j = [tmp_j]
tmp.append([convolutional_layer_output[i:(i+pooling_width),j:(j+pooling_width),kern_indx]/255
for j in range(0,28,8) for i in range(4,28,8)])
tmp_i.append([i for j in range(0,28,8) for i in range(4,28,8)])
tmp_j.append([j for j in range(0,28,8) for i in range(4,28,8)])
tmp[-1] += [convolutional_layer_output[i:(i+pooling_width),j:(j+pooling_width),kern_indx]/255
for i in range(0,28,8) for j in range(4,28,8)]
tmp_i[-1] += [i for i in range(0,28,8) for j in range(4,28,8)]
tmp_j[-1] += [j for i in range(0,28,8) for j in range(4,28,8)]
jmp = [jmp]
jmp_i = [jmp_i]
jmp_j = [jmp_j]
jmp.append([pooling_layer_result[i:(i+1),j:(j+1),kern_indx]/255
for j in range(0,7,2) for i in range(1,7,2)])
jmp_i.append([i for j in range(0,7,2) for i in range(1,7,2)])
jmp_j.append([j for j in range(0,7,2) for i in range(1,7,2)])
jmp[-1] += [pooling_layer_result[i:(i+1),j:(j+1),kern_indx]/255
for i in range(0,7,2) for j in range(1,7,2)]
jmp_i[-1] += [i for i in range(0,7,2) for j in range(1,7,2)]
jmp_j[-1] += [j for i in range(0,7,2) for j in range(1,7,2)]
fig = plotly.subplots.make_subplots(rows=1,cols=2, specs=[[{'type': 'scene'}, {'type': 'scene'}]])
for kern_indx in range(number_kernels):
plot_channel([convolutional_layer_output[...,kern_indx]/255], [3*kern_indx],
'Greys', 0.1, [1,1])
for kern_indx in range(number_kernels):
plot_channel([pooling_layer_result[...,kern_indx]/255], [3*kern_indx],
'Greys', 0.2, [1,2])
for k,kern_indx in enumerate([8,2]):
plot_channel(tmp[k], [3*kern_indx]*len(tmp[k]), px.colors.named_colorscales()[kern_indx],
1, [1,1], i=tmp_i[k], j=tmp_j[k])
plot_channel(jmp[k], [3*kern_indx]*len(jmp[k]), px.colors.named_colorscales()[kern_indx],
1, [1,2], i=jmp_i[k], j=jmp_j[k])
fig.show()
padding='same'?elo to relo but this might not be the best choiceimport numpy as np
mean = np.mean(train_images,axis=(0,1,2,3))
std = np.std(train_images,axis=(0,1,2,3))
train_images = (train_images-mean)/(std+1e-7)
test_images = (test_images-mean)/(std+1e-7)
# Data Augmentation
# https://keras.io/api/preprocessing/image/
from keras.preprocessing.image import ImageDataGenerator
datagen = ImageDataGenerator(rotation_range=15,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True)
datagen.fit(train_images)
# https://appliedmachinelearning.blog/2018/03/24/achieving-90-accuracy-in-object-recognition-task-on-cifar-10-dataset-with-keras-convolutional-neural-networks/
from keras.models import Sequential
from keras.layers import Dense, Activation, Flatten, Dropout, BatchNormalization
from keras.layers import Conv2D, MaxPooling2D
from keras import regularizers
weight_decay = 1e-4
model = Sequential()
# Layer 1: Initial Convolution
model.add(Conv2D(32, (3,3), padding='same', input_shape=(32, 32, 3),
kernel_regularizer=regularizers.l2(weight_decay)))
#model.add(Activation('elu'))
#model.add(BatchNormalization())
model.add(BatchNormalization())
model.add(Activation('relu'))
# Layer 2: Convolution on Initial Convolution followed by Pooling with Dropout
model.add(Conv2D(32, (3,3), padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.2))
# Layer 3: Plain Convolution again
model.add(Conv2D(64, (3,3), padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(BatchNormalization())
model.add(Activation('relu'))
s
# Layer 4: Convolution on Convolution followed by Pooling with Dropout, again
model.add(Conv2D(64, (3,3), padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.3))
# Layer 5: another Plain Convolution
model.add(Conv2D(128, (3,3), padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(BatchNormalization())
model.add(Activation('relu'))
# Layer 6: another Convolution-Pooling-Dropout layer
model.add(Conv2D(128, (3,3), padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.4))
# Layer 7: final
model.add(Flatten())
model.add(layers.Dense(10))
from keras.callbacks import LearningRateScheduler
def lr_schedule(epoch):
lrate = 0.0005
if epoch > 75:
lrate = 0.00025
if epoch > 125:
lrate = 0.00001
return lrate
model.compile(optimizer=tf.keras.optimizers.RMSprop(lr=0.001, decay=1e-6),#'adam',
loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True),#tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
history = model.fit(datagen.flow(train_images, train_labels, batch_size=64),
epochs=150, validation_data=(test_images, test_labels),
callbacks=[LearningRateScheduler(lr_schedule)])
# SPARSE REPRESENTATION WON'T WORK
# WE NEED TO CONVERT TO CATEGORICAL
# https://keras.io/api/preprocessing/image/
# https://appliedmachinelearning.blog/2018/03/24/achieving-90-accuracy-in-object-recognition-task-on-cifar-10-dataset-with-keras-convolutional-neural-networks/
from keras.utils import np_utils
train_labels = np_utils.to_categorical(train_labels, 10)
test_labels = np_utils.to_categorical(test_labels, 10)
zero-padding can keep the network from shrinking with each layer
biases are typically attached according to the natural structures of the convolutional archetecture, but they could also be set to be unique at each locality, for example.
invariance:
variable sized inputs: since convolution is "swept" over or across the input the input shapes don't initially matter in that you can still extract features across the input using a kernel; and using a predefined fixed number of pooling partions with implied regions determined by relative percentages that rescale for different means your pooling function will always produce the same sized output
Cross-correlation (which is actually what is often meant by "convoltution" in NN contexts) is a slight variant of this: $(f*k)(t_0,s_0) = \sum_t \sum_s f(t_0+t,s_0+s) k(t, s)$